Vehicle Detection Project
The goals / steps of this project are the following:
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
from moviepy.editor import VideoFileClip
from IPython.display import HTML
import numpy as np
import pickle
import cv2
import glob
import time
from skimage.feature import hog
from sklearn.svm import LinearSVC
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from scipy.ndimage.measurements import label
%matplotlib inline
print("Completed")
car_images = glob.glob('examples/vehicles/**/*.png')
noncar_images = glob.glob('examples/non-vehicles/**/*.png')
print(len(car_images), len(noncar_images))
print("Completed")
# Showing the car and not car images one by one
for i in range(20):
carimg = cv2.imread(car_images[i])
carimg = cv2.cvtColor(carimg,cv2.COLOR_BGR2RGB)
notcarimg = cv2.imread(noncar_images[i])
notcarimg = cv2.cvtColor(notcarimg,cv2.COLOR_BGR2RGB)
fig, (axs1, axs2) = plt.subplots(1,2, figsize=(16, 16))
axs1.set_title('car')
axs1.imshow(carimg)
axs2.set_title('not car')
axs2.imshow(notcarimg)
print("Completed")
def get_hog_features(img, orient, pix_per_cell, cell_per_block,
vis=False, feature_vec=True):
# Call with two outputs if vis==True
if vis == True:
features, hog_image = hog(img, orientations=orient,
pixels_per_cell=(pix_per_cell, pix_per_cell),
cells_per_block=(cell_per_block, cell_per_block),
transform_sqrt=False,
visualise=vis, feature_vector=feature_vec)
return features, hog_image
# Otherwise call with one output
else:
features = hog(img, orientations=orient,
pixels_per_cell=(pix_per_cell, pix_per_cell),
cells_per_block=(cell_per_block, cell_per_block),
transform_sqrt=False,
visualise=vis, feature_vector=feature_vec)
return features
def bin_spatial(img, size=(32, 32)):
color1 = cv2.resize(img[:,:,0], size).ravel()
color2 = cv2.resize(img[:,:,1], size).ravel()
color3 = cv2.resize(img[:,:,2], size).ravel()
return np.hstack((color1, color2, color3))
def color_hist(img, nbins=32):
# Compute the histogram of the color channels separately
channel1_hist = np.histogram(img[:,:,0], bins=nbins)
channel2_hist = np.histogram(img[:,:,1], bins=nbins)
channel3_hist = np.histogram(img[:,:,2], bins=nbins)
# Concatenate the histograms into a single feature vector
hist_features = np.concatenate((channel1_hist[0], channel2_hist[0], channel3_hist[0]))
return hist_features
print("Completed")
## Tuning Parameters
colorspace = 'YUV' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 11
pix_per_cell = 16
cell_per_block = 2
hog_channel = 'ALL' # Can be 0, 1, 2, or "ALL"
print("Completed")
for i in range(10):
car_img = mpimg.imread(car_images[i])
noncar_img = mpimg.imread(noncar_images[i])
car_feat, car_hog_feat = get_hog_features(car_img[:,:,2], orient, pix_per_cell, cell_per_block, vis=True, feature_vec=True)
noncar_feat, noncar_hog_feat = get_hog_features(noncar_img[:,:,2],orient, pix_per_cell, cell_per_block, vis=True, feature_vec=True)
f, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(20,11))
ax1.imshow(car_img)
ax1.set_title('Car Image')
ax2.imshow(car_hog_feat, cmap='gray')
ax2.set_title('Car HOG')
ax3.imshow(noncar_img)
ax3.set_title('Non-Car Image')
ax4.imshow(noncar_hog_feat, cmap='gray')
ax4.set_title('Non-Car HOG')
print("Completed")
# Define a function to extract features from a list of image locations
def extract_features(imgs, cspace='RGB', orient=9,
pix_per_cell=8, cell_per_block=2, hog_channel=0):
# Create a list to append feature vectors to
features = []
# Iterate through the list of images
for file in imgs:
# Read in each one by one
image = mpimg.imread(file)
# apply color conversion if other than 'RGB'
if cspace != 'RGB':
if cspace == 'HSV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)
elif cspace == 'LUV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)
elif cspace == 'HLS':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)
elif cspace == 'YUV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YUV)
elif cspace == 'YCrCb':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YCrCb)
else: feature_image = np.copy(image)
if hog_channel == 'ALL':
hog_features = []
for channel in range(feature_image.shape[2]):
hog_features.append(get_hog_features(feature_image[:,:,channel],
orient, pix_per_cell, cell_per_block,
vis=False, feature_vec=True))
hog_features = np.ravel(hog_features)
else:
hog_features = get_hog_features(feature_image[:,:,hog_channel], orient,
pix_per_cell, cell_per_block, vis=False, feature_vec=True)
features.append(hog_features)
return features
print("Completed")
t = time.time()
car_features = extract_features(car_images, cspace=colorspace, orient=orient,
pix_per_cell=pix_per_cell, cell_per_block=cell_per_block,
hog_channel=hog_channel)
notcar_features = extract_features(noncar_images, cspace=colorspace, orient=orient,
pix_per_cell=pix_per_cell, cell_per_block=cell_per_block,
hog_channel=hog_channel)
t2 = time.time()
print(round(t2-t, 2), 'Seconds to extract HOG features...')
# Create an array stack of feature vectors
X = np.vstack((car_features, notcar_features)).astype(np.float64)
# Define the labels vector
y = np.hstack((np.ones(len(car_features)), np.zeros(len(notcar_features))))
# Split up data into randomized training and test sets
rand_state = np.random.randint(0, 100)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=rand_state)
print('Using:',orient,'orientations',pix_per_cell,
'pixels per cell and', cell_per_block,'cells per block')
print('Feature vector length:', len(X_train[0]))
print("Completed")
# Use a linear SVC
svc = LinearSVC()
# Check the training time for the SVC
t = time.time()
svc.fit(X_train, y_train)
t2 = time.time()
print(round(t2-t, 2), 'Seconds to train SVC...')
# Check the score of the SVC
print('Test Accuracy of SVC = ', round(svc.score(X_test, y_test), 4))
# Check the prediction time for a single sample
t=time.time()
n_predict = 10
print('My SVC predicts: ', svc.predict(X_test[0:n_predict]))
print('For these',n_predict, 'labels: ', y_test[0:n_predict])
t2 = time.time()
print(round(t2-t, 5), 'Seconds to predict', n_predict,'labels with SVC')
print("Completed")
# Define a single function that can extract features using hog sub-sampling and make predictions
def find_cars(img, ystart, ystop, scale, cspace, hog_channel, svc, X_scaler, orient,
pix_per_cell, cell_per_block, spatial_size, hist_bins):
# array of rectangles where cars were detected
box_list = []
img = img.astype(np.float32)/255
img_tosearch = img[ystart:ystop,:,:]
# apply color conversion if other than 'RGB'
if cspace != 'RGB':
if cspace == 'HSV':
ctrans_tosearch = cv2.cvtColor(img_tosearch, cv2.COLOR_RGB2HSV)
elif cspace == 'LUV':
ctrans_tosearch = cv2.cvtColor(img_tosearch, cv2.COLOR_RGB2LUV)
elif cspace == 'HLS':
ctrans_tosearch = cv2.cvtColor(img_tosearch, cv2.COLOR_RGB2HLS)
elif cspace == 'YUV':
ctrans_tosearch = cv2.cvtColor(img_tosearch, cv2.COLOR_RGB2YUV)
elif cspace == 'YCrCb':
ctrans_tosearch = cv2.cvtColor(img_tosearch, cv2.COLOR_RGB2YCrCb)
else: ctrans_tosearch = np.copy(image)
# rescale image if other than 1.0 scale
if scale != 1:
imshape = ctrans_tosearch.shape
ctrans_tosearch = cv2.resize(ctrans_tosearch, (np.int(imshape[1]/scale), np.int(imshape[0]/scale)))
# select colorspace channel for HOG
if hog_channel == 'ALL':
ch1 = ctrans_tosearch[:,:,0]
ch2 = ctrans_tosearch[:,:,1]
ch3 = ctrans_tosearch[:,:,2]
else:
ch1 = ctrans_tosearch[:,:,hog_channel]
# Define blocks and steps as above
nxblocks = (ch1.shape[1] // pix_per_cell)- cell_per_block + 1
nyblocks = (ch1.shape[0] // pix_per_cell)- cell_per_block + 1
nfeat_per_block = orient*cell_per_block**2
# 64 was the orginal sampling rate, with 8 cells and 8 pix per cell
window = 64
nblocks_per_window = (window // pix_per_cell)- cell_per_block + 1
cells_per_step = 2 # Instead of overlap, define how many cells to step
nxsteps = (nxblocks - nblocks_per_window) // cells_per_step + 1
nysteps = (nyblocks - nblocks_per_window) // cells_per_step + 1
# Compute individual channel HOG features for the entire image
hog1 = get_hog_features(ch1, orient, pix_per_cell, cell_per_block, feature_vec=False)
if hog_channel == 'ALL':
hog2 = get_hog_features(ch2, orient, pix_per_cell, cell_per_block, feature_vec=False)
hog3 = get_hog_features(ch3, orient, pix_per_cell, cell_per_block, feature_vec=False)
for xb in range(nxsteps):
for yb in range(nysteps):
ypos = yb*cells_per_step
xpos = xb*cells_per_step
# Extract HOG for this patch
hog_feat1 = hog1[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
if hog_channel == 'ALL':
hog_feat2 = hog2[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
hog_feat3 = hog3[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
hog_features = np.hstack((hog_feat1, hog_feat2, hog_feat3)).reshape(1,-1)
else:
hog_features = hog_feat1
xleft = xpos*pix_per_cell
ytop = ypos*pix_per_cell
test_prediction = svc.predict(hog_features)
if test_prediction == 1 :
xbox_left = np.int(xleft*scale)
ytop_draw = np.int(ytop*scale)
win_draw = np.int(window*scale)
box_list.append(((xbox_left, ytop_draw+ystart),(xbox_left+win_draw,ytop_draw+win_draw+ystart)))
return box_list
print("Completed")
# Here is your draw_boxes function from the previous exercise
def draw_boxes(img, bboxes, color=(0, 0, 255), thick=6):
# Make a copy of the image
imcopy = np.copy(img)
random_color = False
for bbox in bboxes:
if color == 'random' or random_color:
color = (np.random.randint(0,255), np.random.randint(0,255), np.random.randint(0,255))
random_color = True
cv2.rectangle(imcopy, bbox[0], bbox[1], color, thick)
return imcopy
print("Completed")
test_img = mpimg.imread('./test_images/test1.jpg')
ystart = 400
ystop = 656
scale = 1.5
colorspace = 'YUV' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 11
pix_per_cell = 16
cell_per_block = 2
hog_channel = 'ALL' # Can be 0, 1, 2, or "ALL"
box_list = find_cars(test_img, ystart, ystop, scale, colorspace, hog_channel, svc, None, orient, pix_per_cell, cell_per_block, None, None)
print(len(box_list), 'rectangles found in image')
test_img_rects = draw_boxes(test_img, box_list)
plt.figure(figsize=(10,10))
plt.imshow(test_img_rects)
print("Completed")
Draw the rectangles onto the example image
test_img = mpimg.imread('./test_images/test1.jpg')
box_list = []
ystart = 400
ystop = 464
scale = 1.0
box_list.append(find_cars(test_img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 416
ystop = 480
scale = 1.0
box_list.append(find_cars(test_img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
rectangles = [item for sublist in box_list for item in sublist]
out_img = draw_boxes(test_img, rectangles, color='random', thick=2)
plt.figure(figsize=(10,10))
plt.imshow(out_img)
print('Number of boxes: ', len(box_list))
test_img = mpimg.imread('./test_images/test1.jpg')
box_list = []
colorspace = 'LUV' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 11
pix_per_cell = 16
cell_per_block = 2
hog_channel = 'ALL' # Can be 0, 1, 2, or "ALL"
ystart = 400
ystop = 464
scale = 1.0
box_list.append(find_cars(test_img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 416
ystop = 480
scale = 1.0
box_list.append(find_cars(test_img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 400
ystop = 496
scale = 1.5
box_list.append(find_cars(test_img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 432
ystop = 528
scale = 1.5
box_list.append(find_cars(test_img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 400
ystop = 528
scale = 2.0
box_list.append(find_cars(test_img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 432
ystop = 560
scale = 2.0
box_list.append(find_cars(test_img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 400
ystop = 596
scale = 3.5
box_list.append(find_cars(test_img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 464
ystop = 660
scale = 3.5
box_list.append(find_cars(test_img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
# apparently this is the best way to flatten a list of lists
rectangles = [item for sublist in box_list for item in sublist]
out_img = draw_boxes(test_img, rectangles, color='random', thick=2)
plt.figure(figsize=(10,10))
plt.imshow(out_img)
print("Completed")
def add_heat(heatmap, bbox_list):
# Iterate through list of bboxes
for box in bbox_list:
heatmap[box[0][1]:box[1][1], box[0][0]:box[1][0]] += 1
return heatmap
def apply_threshold(heatmap, threshold):
# Zero out pixels below the threshold
heatmap[heatmap <= threshold] = 0
return heatmap
print("Completed")
# Test out the heatmap
heatmap_img = np.zeros_like(test_img[:,:,0])
heatmap_img = add_heat(heatmap_img, rectangles)
plt.figure(figsize=(16,17))
plt.imshow(heatmap_img, cmap='hot')
print("Completed")
heatmap_img = apply_threshold(heatmap_img, 1)
plt.figure(figsize=(15,17))
plt.imshow(heatmap_img, cmap='hot')
print("Completed")
labels = label(heatmap_img)
plt.figure(figsize=(15,17))
plt.imshow(labels[0], cmap='gray')
print(labels[1], 'cars found')
def draw_labeled_bboxes(img, labels):
# Iterate through all detected cars
rects = []
for car_number in range(1, labels[1]+1):
# Find pixels with each car_number label value
nonzero = (labels[0] == car_number).nonzero()
# Identify x and y values of those pixels
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
# Define a bounding box based on min/max x and y
bbox = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy)))
rects.append(bbox)
# Draw the box on the image
cv2.rectangle(img, bbox[0], bbox[1], (0,0,255), 6)
return img, rects
# Draw bounding boxes on a copy of the image
draw_img, rect = draw_labeled_bboxes(np.copy(test_img), labels)
plt.figure(figsize=(15,17))
plt.imshow(draw_img)
print("Completed")
def process_image(img):
box_list = []
colorspace = 'YUV' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 11
pix_per_cell = 16
cell_per_block = 2
hog_channel = 'ALL' # Can be 0, 1, 2, or "ALL"
ystart = 400
ystop = 464
scale = 1.0
box_list.append(find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 416
ystop = 480
scale = 1.0
box_list.append(find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 400
ystop = 496
scale = 1.5
box_list.append(find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 432
ystop = 528
scale = 1.5
box_list.append(find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 400
ystop = 528
scale = 2.0
box_list.append(find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 432
ystop = 560
scale = 2.0
box_list.append(find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 400
ystop = 596
scale = 3.5
box_list.append(find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 464
ystop = 660
scale = 3.5
box_list.append(find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
rectangles = [item for sublist in box_list for item in sublist]
heatmap_img = np.zeros_like(img[:,:,0])
heatmap_img = add_heat(heatmap_img, rectangles)
heatmap_img = apply_threshold(heatmap_img, 1)
labels = label(heatmap_img)
draw_img, rects = draw_labeled_bboxes(np.copy(img), labels)
return draw_img
print("Completed")
test_images = glob.glob('./test_images/test*.jpg')
for i, im in enumerate(test_images):
image = mpimg.imread(im)
image = process_image(image)
fig, axs = plt.subplots(1, 1, figsize=(15,17))
axs.imshow(image)
test_out_file = 'test_video_out.mp4'
clip_test = VideoFileClip('test_video.mp4')
clip_test_out = clip_test.fl_image(process_image)
%time clip_test_out.write_videofile(test_out_file, audio=False)
# Define a class to store data from video
class Vehicle_Detect():
def __init__(self):
# history of rectangles previous n frames
self.save_rects = []
def add_rects(self, rects):
self.save_rects.append(rects)
if len(self.save_rects) > 10:
# throw out oldest rectangle set(s)
self.save_rects = self.save_rects[len(self.save_rects)-10:]
det = Vehicle_Detect()
def process_image_video(img):
box_list = []
colorspace = 'YUV' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 11
pix_per_cell = 16
cell_per_block = 2
hog_channel = 'ALL' # Can be 0, 1, 2, or "ALL"
ystart = 400
ystop = 464
scale = 1.0
box_list.append(find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 416
ystop = 480
scale = 1.0
box_list.append(find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 400
ystop = 496
scale = 1.5
box_list.append(find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 432
ystop = 528
scale = 1.5
box_list.append(find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 400
ystop = 528
scale = 2.0
box_list.append(find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 432
ystop = 560
scale = 2.0
box_list.append(find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 400
ystop = 596
scale = 3.5
box_list.append(find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
ystart = 464
ystop = 660
scale = 3.5
box_list.append(find_cars(img, ystart, ystop, scale, colorspace, hog_channel, svc, None,
orient, pix_per_cell, cell_per_block, None, None))
rectangles = [item for sublist in box_list for item in sublist]
if len(rectangles) > 0:
det.add_rects(rectangles)
heatmap_img = np.zeros_like(img[:,:,0])
for rect in det.save_rects:
heatmap_img = add_heat(heatmap_img, rect)
heatmap_img = apply_threshold(heatmap_img, 1 + len(det.save_rects)//2)
labels = label(heatmap_img)
draw_img, rect = draw_labeled_bboxes(np.copy(img), labels)
return draw_img
print("Completed")
test_out_file2 = 'test_video_out.mp4'
clip_test2 = VideoFileClip('test_video.mp4')
clip_test_out2 = clip_test2.fl_image(process_image_video)
%time clip_test_out2.write_videofile(test_out_file2, audio=False)
proj_out_file = 'project_video_out.mp4'
clip_proj = VideoFileClip('project_video.mp4')
clip_proj_out = clip_proj.fl_image(process_image_video)
%time clip_proj_out.write_videofile(proj_out_file, audio=False)